{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window.Plotly) {{require(['plotly'],function(plotly) {window.Plotly=plotly;});}}</script>"
      ],
      "text/vnd.plotly.v1+html": [
       "<script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window.Plotly) {{require(['plotly'],function(plotly) {window.Plotly=plotly;});}}</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python version: 3.6.3 |Anaconda custom (64-bit)| (default, Oct 15 2017, 03:27:45) [MSC v.1900 64 bit (AMD64)]\n",
      "pandas version: 0.20.3\n",
      "NumPy version: 1.14.3\n",
      "SciPy version: 1.1.0\n",
      "scikit-learn version: 0.19.1\n",
      "matplotlib version: 2.1.0\n",
      "IPython version: 6.1.0\n",
      "Using Hard-Coded Configuration for data_file_path\n",
      "data_file_path:  C:/Development/kaggle--home-credit-default-risk/data/\n",
      "data_file_name:  application_train.csv\n"
     ]
    }
   ],
   "source": [
    "%run NB01-Load.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Secondary / secondary special    218391\n",
       "Higher education                  74863\n",
       "Incomplete higher                 10277\n",
       "Lower secondary                    3816\n",
       "Academic degree                     164\n",
       "Name: NAME_EDUCATION_TYPE, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['NAME_EDUCATION_TYPE'].value_counts(dropna=False, sort=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Secondary / secondary special    0.710189\n",
       "Higher education                 0.243448\n",
       "Incomplete higher                0.033420\n",
       "Lower secondary                  0.012409\n",
       "Academic degree                  0.000533\n",
       "Name: NAME_EDUCATION_TYPE, dtype: float64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1.00 * df['NAME_EDUCATION_TYPE'].value_counts() / len(df['NAME_EDUCATION_TYPE'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Secondary / secondary special    71.018923\n",
       "Higher education                 24.344820\n",
       "Incomplete higher                 3.341994\n",
       "Lower secondary                   1.240931\n",
       "Academic degree                   0.053331\n",
       "Name: NAME_EDUCATION_TYPE, dtype: float64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "100.00 * df['NAME_EDUCATION_TYPE'].value_counts() / len(df['NAME_EDUCATION_TYPE'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(application_train.select_dtypes(include=['bool']).columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(application_train.select_dtypes(include=['int64']).columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(application_train.select_dtypes(include=['float64']).columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(application_train.select_dtypes(include=['O']).columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# List of unique values in the df['name'] column\n",
    "# df.name.unique()\n",
    "application_train.TARGET.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train['TARGET'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of unique values in the df['name'] column\n",
    "application_train.TARGET.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train['TARGET'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train['TARGET'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://stackoverflow.com/questions/45759966/counting-unique-values-in-a-column-in-pandas-dataframe-like-in-qlik\n",
    "\n",
    "Count distict values, use nunique:\n",
    "df['hID'].nunique()\n",
    "Count only non-null values, use count:\n",
    "df['hID'].count()\n",
    "Count total values including null values, use size attribute:\n",
    "df['hID'].size\n",
    "To add condition...\n",
    "Use boolean indexing:\n",
    "df.loc[df['mID']=='A','hID'].agg(['nunique','count','size'])\n",
    "Or, using query:\n",
    "df.query('mID == \"A\"')['hID'].agg(['nunique','count','size'])\n",
    "\n",
    "New in pandas 0.20.0 pd.DataFrame.agg\n",
    "df.agg(['count', 'size', 'nunique'])\n",
    "You've always been able to do an agg within a groupby. I used stack at the end because I like the presentation better.\n",
    "df.groupby('mID').agg(['count', 'size', 'nunique']).stack()\n",
    "\n",
    "\n",
    "https://stackoverflow.com/questions/45125408/how-to-count-the-distinct-values-across-a-column-in-pandas\n",
    "\n",
    "df[['Company', 'Date']].drop_duplicates()['Company'].value_counts()\n",
    "df.groupby('Company')['Date'].nunique()\n",
    "\n",
    "\n",
    "https://stackoverflow.com/questions/48162201/pandas-number-of-unique-values-and-sort-by-the-number-of-unique\n",
    "\n",
    "df = df.groupby('A')['B'].nunique().sort_values(ascending=False).reset_index(name='count')\n",
    "print (df)\n",
    "\n",
    "\n",
    "https://stackoverflow.com/questions/38309729/count-unique-values-with-pandas-per-groups/38309823\n",
    "\n",
    "You can retain the column name like this:\n",
    "df = df.groupby(by='domain', as_index=False).agg({'ID': pd.Series.nunique})\n",
    "The difference is that 'nunique()' returns a Series and 'agg()' returns a DataFrame.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for column_name in application_train_column_names:\n",
    "    number_of_unique_values = application_train[column_name].nunique()\n",
    "    print ('Column Name: ', column_name, 'Number of Unique Values: ', number_of_unique_values, 'Cardinality: ', round(number_of_unique_values/application_train__row_count,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict((column_name,None) for column_name in application_train_column_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict((column_name,application_train__row_count) for column_name in application_train_column_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://stackoverflow.com/questions/3869487/how-do-i-create-a-dictionary-with-keys-from-a-list-and-values-defaulting-to-say\n",
    "# Generator expressions avoid the memory overhead of populating the whole list.\n",
    "# https://stackoverflow.com/questions/2241891/how-to-initialize-a-dict-with-keys-from-a-list-and-empty-value-in-python\n",
    "# dict.fromkeys(keys_list)\n",
    "# Be careful with initializing to something mutable: If you call, e.g., dict.fromkeys([1, 2, 3], []), all of the keys are mapped to the same list, and modifying one will modify them all.\n",
    "# \n",
    "# dict-comprehension solution\n",
    "# keys = [1,2,3,5,6,7]\n",
    "# {key: None for key in keys}\n",
    "#> {1: None, 2: None, 3: None, 5: None, 6: None, 7: None}\n",
    "# \n",
    "# Using a dict-comp also allows the value to be the result of calling a function\n",
    "#   (which could be passed the key as an argument, if desired)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "# http://pbpython.com/pandas-list-dict.html\n",
    "The “default” manner to create a DataFrame from python is to use a list of dictionaries.\n",
    "In this case each dictionary key is used for the column headings.\n",
    "A default index will be created automatically:\n",
    "sales = [{'account': 'Jones LLC', 'Jan': 150, 'Feb': 200, 'Mar': 140},\n",
    "         {'account': 'Alpha Co',  'Jan': 200, 'Feb': 210, 'Mar': 215},\n",
    "         {'account': 'Blue Inc',  'Jan': 50,  'Feb': 90,  'Mar': 95 }]\n",
    "df = pd.DataFrame(sales)\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "empty_dictionary = {}\n",
    "empty_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "any(empty_dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(empty_dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bool(empty_dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "falsy_dictionary = {0:False}\n",
    "falsy_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "any(falsy_dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(falsy_dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bool(falsy_dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "# https://stackoverflow.com/questions/23177439/python-checking-if-a-dictionary-is-empty-doesnt-seem-to-work\n",
    "test_dict = {}\n",
    "\n",
    "# Option 1\n",
    "if not test_dict:\n",
    "    print \"Dict is Empty\"\n",
    "\n",
    "# Option 2\n",
    "if not bool(test_dict):\n",
    "    print \"Dict is Empty\"\n",
    "\n",
    "# Option 3\n",
    "if len(test_dict) == 0:\n",
    "    print \"Dict is Empty\"\n",
    "\n",
    "# The first test in the answer above is true not only if the dict exists and is empty, but also if test_dict is None.\n",
    "# So use this test only when you know that the dict object exists (or when the difference does not matter).\n",
    "# The second way also has that behavior.\n",
    "# Only the third way barks if test_dict is None.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train_as_objects = pd.read_table(path + 'application_train.csv', sep=',', dtype=object)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train_column_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://stackoverflow.com/questions/18667410/how-can-i-check-if-a-string-only-contains-letters-in-python\n",
    "\n",
    "if string.isalpha():\n",
    "    print(\"It's all letters\")\n",
    "\n",
    "# NOTE: using [A-Za-z ]+ will not match names with non ASCII letterss\n",
    "# NOTE: using \\w includes digits\n",
    "import re\n",
    "def only_letters(tested_string):\n",
    "    #match = re.match(\"^[ABCDEFGHJKLM]*$\", tested_string)\n",
    "    #match = re.match(\"^[A-HJ-M]*$\", tested_string)\n",
    "    match = re.match(\"^[A-Za-z]*$\", tested_string)\n",
    "    return match is not None\n",
    "\n",
    "import re\n",
    "def only_letters(string):\n",
    "    return re.match(r'[a-z\\s]+$',string,2) # JWB: What's with the \",2\"?\n",
    "\n",
    "def only_letters(string):\n",
    "    return all(letter.isalpha() for letter in string)\n",
    "\n",
    "def only_letters(s):\n",
    "    for c in s:\n",
    "        cat = unicodedata.category(c)\n",
    "        # Lu: Category: Letter, Uppercase (https://codepoints.net/search?gc=Lu)\n",
    "        # Ll: Category: Letter, Lowercase (https://codepoints.net/search?gc=Ll)\n",
    "        # Lt: Category: Letter, Titlecase (https://codepoints.net/search?gc=Lt)\n",
    "        # Lm: Category: Letter, Modifier  (https://codepoints.net/search?gc=Lm)\n",
    "        # Lo: Category: Letter, Other     (https://codepoints.net/search?gc=Lo)\n",
    "        # Latin-1: There are no Lm or Lt category codepoints in the Latin-1 subset of Unicode and only 2 Lo characters, ª (U+00AA) and º (U+00BA), Feminine and Masculine Ordinal Indicator).\n",
    "        if cat not in ('Lu','Ll','Lo'):\n",
    "            return False\n",
    "    return True\n",
    "\n",
    "# https://stackoverflow.com/questions/29460405/checking-if-string-is-only-letters-and-spaces-python\n",
    "# To require that the string contains only alphas and spaces:\n",
    "if all(x.isalpha() or x.isspace() for x in string):\n",
    "    print(\"Only alphabetical letters and spaces: yes\")\n",
    "else:\n",
    "    print(\"Only alphabetical letters and spaces: no\")\n",
    "# To require that the string contains at least one alpha and at least one space:\n",
    "if any(x.isalpha() for x in string) and any(x.isspace() for x in string):\n",
    "# To require that the string contains at least one alpha, at least one space, and only alphas and spaces:\n",
    "if (any(x.isalpha() for x in string)\n",
    "    and any(x.isspace() for x in string)\n",
    "    and all(x.isalpha() or x.isspace() for x in string)):\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#locale.getlocale()\n",
    "#> NameError: name 'locale' is not defined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'dog'.isalpha()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'äöå'.isalpha()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'привіт'.isalpha()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "u'привіт'.isalpha()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "repr('äöå')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ascii('äöå')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://stackoverflow.com/questions/4286637/python-isalpha-and-scandics\n",
    "# http://en.wikipedia.org/wiki/Windows-1252\n",
    "s = '\\xe4\\xf6\\xe5'\n",
    "import unicodedata\n",
    "for c in s:\n",
    "    u = c.decode('1252')\n",
    "    print (ascii(c), ascii(u), unicodedata.name(u, '<no name>'))\n",
    "#'\\xe4' u'\\xe4' LATIN SMALL LETTER A WITH DIAERESIS\n",
    "#'\\xf6' u'\\xf6' LATIN SMALL LETTER O WITH DIAERESIS\n",
    "#'\\xe5' u'\\xe5' LATIN SMALL LETTER A WITH RING ABOVE\n",
    "s.isalpha()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The isalpha() methods returns “True” if all characters in the string are alphabets, Otherwise, It returns “False”.\n",
    "# This function is used to check if the argument contains any alphabets characters such as:\n",
    "#     ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz\n",
    "# Both uppercase and lowercase alphabets return “True”\n",
    "# Space is not considered to be alphabet, therefore it returns “False”\n",
    "\n",
    "def is_plain_text(string):\n",
    "    if string != 0:\n",
    "            # require that the string contains only alphas\n",
    "            #if all(character.isalpha() for character in string)\n",
    "            # require that the string contains only alphas and spaces\n",
    "            if all(character.isalpha() or character.isspace() for character in string):\n",
    "                return True\n",
    "            else:\n",
    "                return False\n",
    "    else:\n",
    "            return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "is_plain_text(application_train_as_objects['SK_ID_CURR'])\n",
    "#TypeError: 'numpy.bool_' object is not iterable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train_as_objects['SK_ID_CURR'].dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train_as_objects['SK_ID_CURR'].str.isdigit().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train_as_objects['SK_ID_CURR'].str.isdigit().all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train_as_objects['SK_ID_CURR'].str.isalpha().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train_as_objects['SK_ID_CURR'].str.isalpha().all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[pd.to_numeric(df.A, errors='coerce').notnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train['SK_ID_CURR'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train['SK_ID_CURR'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# checking for uniqueness:\n",
    "print(len(application_train['SK_ID_CURR'].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# How many classes\n",
    "application_train['TARGET'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Distribution of those classes\n",
    "application_train['TARGET'].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtypes = application_train.dtypes\n",
    "dtypes = dtypes[dtypes != 'object']\n",
    "features = list(set(dtypes.index) - set(['TARGET']))\n",
    "len(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df[features]\n",
    "y = df['TARGET']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "id_column = ['SK_ID_CURR']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_column = ['TARGET']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "boolean_columns = ['FLAG_OWN_CAR', 'FLAG_OWN_REALTY', 'FLAG_MOBIL', 'FLAG_EMP_PHONE', 'FLAG_WORK_PHONE', 'FLAG_CONT_MOBILE', 'FLAG_PHONE', 'FLAG_EMAIL', 'REG_REGION_NOT_LIVE_REGION', 'REG_REGION_NOT_WORK_REGION', 'LIVE_REGION_NOT_WORK_REGION', 'REG_CITY_NOT_LIVE_CITY', 'REG_CITY_NOT_WORK_CITY', 'LIVE_CITY_NOT_WORK_CITY', 'FLAG_DOCUMENT_2', 'FLAG_DOCUMENT_3', 'FLAG_DOCUMENT_4', 'FLAG_DOCUMENT_5', 'FLAG_DOCUMENT_6', 'FLAG_DOCUMENT_7', 'FLAG_DOCUMENT_8', 'FLAG_DOCUMENT_9', 'FLAG_DOCUMENT_10', 'FLAG_DOCUMENT_11', 'FLAG_DOCUMENT_12', 'FLAG_DOCUMENT_13', 'FLAG_DOCUMENT_14', 'FLAG_DOCUMENT_15', 'FLAG_DOCUMENT_16', 'FLAG_DOCUMENT_17', 'FLAG_DOCUMENT_18', 'FLAG_DOCUMENT_19', 'FLAG_DOCUMENT_20', 'FLAG_DOCUMENT_21']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical_columns = ['NAME_CONTRACT_TYPE', 'CODE_GENDER', 'NAME_TYPE_SUITE', 'NAME_INCOME_TYPE', 'NAME_EDUCATION_TYPE', 'NAME_FAMILY_STATUS', 'NAME_HOUSING_TYPE', 'OCCUPATION_TYPE', 'WEEKDAY_APPR_PROCESS_START', 'ORGANIZATION_TYPE', 'FONDKAPREMONT_MODE', 'HOUSETYPE_MODE', 'WALLSMATERIAL_MODE', 'EMERGENCYSTATE_MODE']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "object_columns = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "numerical_columns = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "int_columns = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "float_columns = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "datetime_columns = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "date_columns = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nondata_columns = ['SK_ID_CURR']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_distribution = application_train['TARGET'].value_counts()\n",
    "target_distribution.plot.pie(figsize=(10, 10),\n",
    "                             title='Target Distribution',\n",
    "                             fontsize=15, \n",
    "                             legend=True,\n",
    "                             autopct=lambda v: \"{:0.1f}%\".format(v))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nan_precents = (application_train.isnull().sum()/application_train.isnull().count()*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_overview_df  = pd.concat([total_nans, nan_precents], axis=1, keys=['NaN Count', 'NaN Pencent'])\n",
    "feature_overview_df['Type'] = [application_train[c].dtype for c in feature_overview_df.index]\n",
    "pd.set_option('display.max_rows', None)\n",
    "display(feature_overview_df)\n",
    "pd.set_option('display.max_rows', 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_application_is_nan_df = pd.DataFrame()\n",
    "for column in application_train.columns:\n",
    "    if application_train[column].isnull().sum() == 0:\n",
    "        continue\n",
    "    all_application_is_nan_df['is_nan_' + column] = application_train[column].isnull()\n",
    "    all_application_is_nan_df['is_nan_' + column] = all_application_is_nan_df['is_nan_' + column].map(lambda v: 1 if v else 0)\n",
    "all_application_is_nan_df['target'] = application_train['TARGET']\n",
    "all_application_is_nan_df = all_application_is_nan_df[pd.notnull(all_application_is_nan_df['target'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display(all_application_is_nan_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y = all_application_is_nan_df.pop('target')\n",
    "X = all_application_is_nan_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X, valid_X, train_Y, valid_Y = train_test_split(X, Y, test_size=0.2, random_state=2018)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = LGBMClassifier(n_estimators=200, learning_rate=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf.fit(\n",
    "        train_X,\n",
    "        train_Y,\n",
    "        eval_set=[(train_X, train_Y), (valid_X, valid_Y)],\n",
    "        eval_metric='auc',\n",
    "        early_stopping_rounds=50,\n",
    "        verbose=False\n",
    "       )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_importance(clf, figsize=(10,10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(\"only showing the distribution for the first few columns, edit the counter to show all distribution\")\n",
    "#show_feature_count = 10\n",
    "#for column in all_application_df.columns:\n",
    "#   if show_feature_count == 0:\n",
    "#        break\n",
    "#    show_feature_count -= 1\n",
    "#    draw_feature_distribution(all_application_df, column)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "draw_feature_distribution(application_train, 'TARGET')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "draw_feature_distribution(application_train, 'DAYS_EMPLOYED')\n",
    "# ToDo(JamesBalcomb): fix \"ValueError: max() arg is an empty sequence\" - add check for 'class_t_values'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#EXT_SOURCE_1\n",
    "#EXT_SOURCE_2\n",
    "#EXT_SOURCE_3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "draw_feature_distribution(application_train, 'EXT_SOURCE_1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "draw_feature_distribution(application_train, 'EXT_SOURCE_2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "draw_feature_distribution(application_train, 'EXT_SOURCE_3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# EXT_SOURCE_1, EXT_SOURCE_2, EXT_SOURCE_3\n",
    "\n",
    "Q: Is there a relationship between any of these three continuous variables and the binary classification target variable?\n",
    "A: Yes, but EXT_SOURCE_2 is oddly shapen."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Seaborn Violin Plot - correlation; distribution and density\n",
    "sns.violinplot(x='TARGET', y='EXT_SOURCE_1', data=application_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Seaborn Violin Plot - correlation; distribution and density\n",
    "sns.violinplot(x='TARGET', y='EXT_SOURCE_2', data=application_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Seaborn Violin Plot - correlation; distribution and density\n",
    "sns.violinplot(x='TARGET', y='EXT_SOURCE_3', data=application_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train.hist(column='EXT_SOURCE_1', # Column to plot\n",
    "              figsize=(8,8),                  # Plot size\n",
    "              color=\"blue\"                    # Plot color\n",
    "              )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train.hist(column='EXT_SOURCE_2', # Column to plot\n",
    "              figsize=(8,8),                  # Plot size\n",
    "              color=\"blue\"                    # Plot color\n",
    "              )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train.hist(column='EXT_SOURCE_3', # Column to plot\n",
    "              figsize=(8,8),                  # Plot size\n",
    "              color=\"blue\"                    # Plot color\n",
    "              )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://gist.github.com/ltfschoen/4c5d2cf26b8be5355043273493a6b8b9#file-proportions_of_missing_data_in_dataframe_columns-py\n",
    "def get_percentage_missing(series):\n",
    "    \"\"\" Calculates percentage of NaN values in DataFrame\n",
    "    :param series: Pandas DataFrame object\n",
    "    :return: float\n",
    "    \"\"\"\n",
    "    num = series.isnull().sum()\n",
    "    den = len(series)\n",
    "    return round(num / den, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_percentage_missing(application_train['EXT_SOURCE_1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_percentage_missing(application_train['EXT_SOURCE_2'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_percentage_missing(application_train['EXT_SOURCE_3'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # https://datascience.stackexchange.com/questions/12645/how-to-count-the-number-of-missing-values-in-each-row-in-pandas-dataframe\n",
    "# # Count of Missing Values per Column\n",
    "# df.isnull().sum(axis=0)\n",
    "# # Count of Missing Values per Row\n",
    "# df.isnull().sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://towardsdatascience.com/the-tale-of-missing-values-in-python-c96beb0e8a9d\n",
    "# If the missing value isn’t identified as NaN , then we have to first convert or replace such non NaN entry with a NaN.\n",
    "data_name[‘column_name’].replace(0, np.nan, inplace= True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://gist.github.com/ltfschoen/4c5d2cf26b8be5355043273493a6b8b9#file-proportions_of_missing_data_in_dataframe_columns-py\n",
    "df = application_train\n",
    "# Only include columns that contain any NaN values\n",
    "df_with_any_null_values = df[df.columns[df.isnull().any()].tolist()]\n",
    "\n",
    "get_percentage_missing(df_with_any_null_values)\n",
    "\n",
    "# Iterate over columns in DataFrame and delete those with where >30% of the values are null/NaN\n",
    "for name, values in df_with_any_null_values.iteritems():\n",
    "    # print(\"%r: %r\" % (name, values))\n",
    "    if get_percentage_missing(df_with_any_null_values[name]) > 0.30:\n",
    "        print(\"Deleting Column %r: \" % (name))\n",
    "        # df_with_any_null_values.drop(name, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12,5))\n",
    "plt.title(\"Distribution of EXT_SOURCE_1\")\n",
    "ax = sns.distplot(application_train[\"EXT_SOURCE_1\"].dropna())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12,5))\n",
    "plt.title(\"Distribution of EXT_SOURCE_2\")\n",
    "ax = sns.distplot(application_train[\"EXT_SOURCE_2\"].dropna())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12,5))\n",
    "plt.title(\"Distribution of EXT_SOURCE_3\")\n",
    "ax = sns.distplot(application_train[\"EXT_SOURCE_3\"].dropna())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train['EXT_SOURCE_AVG'] = application_train[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].mean(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12,5))\n",
    "plt.title(\"Distribution of EXT_SOURCE_AVG\")\n",
    "ax = sns.distplot(application_train[\"EXT_SOURCE_AVG\"].dropna())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://stackoverflow.com/questions/35277075/python-pandas-counting-the-occurrences-of-a-specific-value\n",
    "#df.loc[df.education == '9th', 'education'].count()\n",
    "#(df.education == '9th').sum()\n",
    "#df.query('education == \"9th\"').education.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train.loc[application_train.EXT_SOURCE_1 == 0.0, 'EXT_SOURCE_1'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train.loc[application_train.EXT_SOURCE_2 == 0.0, 'EXT_SOURCE_2'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train.loc[application_train.EXT_SOURCE_3 == 0.0, 'EXT_SOURCE_3'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train['EXT_SOURCE_1'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train['EXT_SOURCE_2'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train['EXT_SOURCE_3'].value_counts()\n",
    "# application_train['EXT_SOURCE_3'].value_counts()[:20]\n",
    "# # ValueError: index must be monotonic increasing or decreasing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train['EXT_SOURCE_1'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train['EXT_SOURCE_2'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train['EXT_SOURCE_3'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train['EXT_SOURCE_1'].plot(kind='hist', bins=[0.0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train['EXT_SOURCE_1'].plot(kind='hist', bins=[0.0,0.2,0.4,0.6,0.8,1.0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train['EXT_SOURCE_2'].plot(kind='hist', bins=[0.0,0.2,0.4,0.6,0.8,1.0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train['EXT_SOURCE_3'].plot(kind='hist', bins=[0.0,0.2,0.4,0.6,0.8,1.0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://community.modeanalytics.com/python/tutorial/python-histograms-boxplots-and-distributions/\n",
    "bin_values = np.arange(start=0, stop=1, step=0.2)\n",
    "us_mq_airlines_index = application_train['TARGET'].isin(['US','MQ']) # create index\n",
    "us_mq_airlines = application_train[us_mq_airlines_index] # select rows\n",
    "group_carriers = us_mq_airlines.groupby('TARGET')['EXT_SOURCE_1'] # group values\n",
    "group_carriers.plot(kind='hist', bins=bin_values, figsize=[12,6], alpha=.4, legend=True) # alpha for transparency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train['EXT_SOURCE_1'].plot(kind='box', figsize=[16,8])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train['EXT_SOURCE_2'].plot(kind='box', figsize=[16,8])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train['EXT_SOURCE_3'].plot(kind='box', figsize=[16,8])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a function to create the scatterplot. This makes it easy to\n",
    "# reuse code within and across notebooks\n",
    "def scatterplot(x_data, y_data, x_label, y_label, title):\n",
    "\n",
    "    # Create the plot object\n",
    "    _, ax = plt.subplots()\n",
    "\n",
    "    # Plot the data, set the size (s), color and transparency (alpha) of the points\n",
    "    ax.scatter(x_data, y_data, s = 30, color = '#539caf', alpha = 0.75)\n",
    "\n",
    "    # Label the axes and provide a title\n",
    "    ax.set_title(title)\n",
    "    ax.set_xlabel(x_label)\n",
    "    ax.set_ylabel(y_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Call the function to create plot\n",
    "#scatterplot(x_data = daily_data['temp']\n",
    "#            , y_data = daily_data['cnt']\n",
    "#            , x_label = 'Normalized temperature (C)'\n",
    "#            , y_label = 'Check outs'\n",
    "#            , title = 'Number of Check Outs vs Temperature')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Call the function to create plot\n",
    "scatterplot(x_data = application_train['EXT_SOURCE_1']\n",
    "            , y_data = application_train['EXT_SOURCE_2']\n",
    "            , x_label = 'EXT_SOURCE_1'\n",
    "            , y_label = 'EXT_SOURCE_2'\n",
    "            , title = 'EXT_SOURCE_1 vs. EXT_SOURCE_2'\n",
    "           )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Call the function to create plot\n",
    "scatterplot(x_data = application_train['EXT_SOURCE_1']\n",
    "            , y_data = application_train['EXT_SOURCE_3']\n",
    "            , x_label = 'EXT_SOURCE_1'\n",
    "            , y_label = 'EXT_SOURCE_2'\n",
    "            , title = 'EXT_SOURCE_1 vs. EXT_SOURCE_3'\n",
    "           )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Call the function to create plot\n",
    "scatterplot(x_data = application_train['EXT_SOURCE_2']\n",
    "            , y_data = application_train['EXT_SOURCE_3']\n",
    "            , x_label = 'EXT_SOURCE_2'\n",
    "            , y_label = 'EXT_SOURCE_3'\n",
    "            , title = 'EXT_SOURCE_2 vs. EXT_SOURCE_3'\n",
    "           )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train__ext_source_1__dropna = application_train.loc[:,['EXT_SOURCE_1','TARGET']]\n",
    "application_train__ext_source_1__dropna.dropna(inplace = True)\n",
    "scipy.stats.pointbiserialr(application_train__ext_source_1__dropna['EXT_SOURCE_1'], application_train__ext_source_1__dropna['TARGET'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train__ext_source_2__dropna = application_train.loc[:,['EXT_SOURCE_2','TARGET']]\n",
    "application_train__ext_source_2__dropna.dropna(inplace = True)\n",
    "scipy.stats.pointbiserialr(application_train__ext_source_2__dropna['EXT_SOURCE_2'], application_train__ext_source_2__dropna['TARGET'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train__ext_source_3__dropna = application_train.loc[:,['EXT_SOURCE_3','TARGET']]\n",
    "application_train__ext_source_3__dropna.dropna(inplace = True)\n",
    "scipy.stats.pointbiserialr(application_train__ext_source_3__dropna['EXT_SOURCE_3'], application_train__ext_source_3__dropna['TARGET'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scipy.stats.pearsonr(application_train__ext_source_1__dropna['EXT_SOURCE_1'], application_train__ext_source_1__dropna['TARGET'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scipy.stats.pearsonr(application_train__ext_source_2__dropna['EXT_SOURCE_2'], application_train__ext_source_2__dropna['TARGET'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scipy.stats.pearsonr(application_train__ext_source_3__dropna['EXT_SOURCE_3'], application_train__ext_source_3__dropna['TARGET'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.corrcoef(application_train__ext_source_1__dropna['EXT_SOURCE_1'], application_train__ext_source_1__dropna['TARGET'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.corrcoef(application_train__ext_source_2__dropna['EXT_SOURCE_2'], application_train__ext_source_2__dropna['TARGET'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.corrcoef(application_train__ext_source_3__dropna['EXT_SOURCE_3'], application_train__ext_source_3__dropna['TARGET'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = previous_application[\"NAME_CONTRACT_TYPE\"].value_counts()\n",
    "fig = {\n",
    "  \"data\": [\n",
    "    {\n",
    "      \"values\": temp.values,\n",
    "      \"labels\": temp.index,\n",
    "      \"domain\": {\"x\": [0, .48]},\n",
    "      #\"name\": \"Types of Loans\",\n",
    "      #\"hoverinfo\":\"label+percent+name\",\n",
    "      \"hole\": .7,\n",
    "      \"type\": \"pie\"\n",
    "    },\n",
    "    \n",
    "    ],\n",
    "  \"layout\": {\n",
    "        \"title\":\"Contract product type of previous application\",\n",
    "        \"annotations\": [\n",
    "            {\n",
    "                \"font\": {\n",
    "                    \"size\": 20\n",
    "                },\n",
    "                \"showarrow\": False,\n",
    "                \"text\": \"Contract product type\",\n",
    "                \"x\": 0.12,\n",
    "                \"y\": 0.5\n",
    "            }\n",
    "            \n",
    "        ]\n",
    "    }\n",
    "}\n",
    "iplot(fig, filename='donut')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pearson Correlation of features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = [\n",
    "    go.Heatmap(\n",
    "        z=application_train.corr().values,\n",
    "        x=application_train.columns.values,\n",
    "        y=application_train.columns.values,\n",
    "        colorscale='Viridis',\n",
    "        reversescale = False,\n",
    "        text = True ,\n",
    "        opacity = 1.0 )\n",
    "]\n",
    "\n",
    "layout = go.Layout(\n",
    "    title='Pearson Correlation of features',\n",
    "    xaxis = dict(ticks='', nticks=36),\n",
    "    yaxis = dict(ticks='' ),\n",
    "    width = 900, height = 700,\n",
    "margin=dict(\n",
    "    l=240,\n",
    "),)\n",
    "\n",
    "fig = go.Figure(data=data, layout=layout)\n",
    "py.iplot(fig, filename='labelled-heatmap')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Multicollinearity\n",
    "https://stackoverflow.com/questions/25676145/capturing-high-multi-collinearity-in-statsmodels/44012251#44012251\n",
    "https://stackoverflow.com/questions/25676145/capturing-high-multi-collinearity-in-statsmodels/25833792#25833792\n",
    "https://onlinecourses.science.psu.edu/stat501/node/347/\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## https://stackoverflow.com/questions/25676145/capturing-high-multi-collinearity-in-statsmodels/44012251#44012251\n",
    "##...looking for a single number that captured the collinearity\n",
    "##...options include the determinant and condition number of the correlation matrix\n",
    "##...determinant of the correlation matrix will \"range from 0 (Perfect Collinearity) to 1 (No Collinearity)\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute correlation matrices\n",
    "pearson_product_moment_correlation_coefficients = np.corrcoef(application_train, rowvar=0)\n",
    "## https://docs.scipy.org/doc/numpy/reference/generated/numpy.corrcoef.html\n",
    "## Return Pearson product-moment correlation coefficients."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compare the determinants\n",
    "print np.linalg.det(pearson_product_moment_correlation_coefficients)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the condition number of the covariance matrix will approach infinity with perfect linear dependence\n",
    "print np.linalg.cond(pearson_product_moment_correlation_coefficients)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train.info(memory_usage='deep')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train.memory_usage()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "application_train.memory_usage(index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://towardsdatascience.com/automated-feature-engineering-in-python-99baf11cc219\n",
    "# Group loans by client id and calculate mean, max, min of loans\n",
    "stats = loans.groupby('client_id')['loan_amount'].agg(['mean', 'max', 'min'])\n",
    "stats.columns = ['mean_loan_amount', 'max_loan_amount', 'min_loan_amount']\n",
    "\n",
    "# Merge with the clients dataframe\n",
    "stats = clients.merge(stats, left_on = 'client_id', right_index=True, how = 'left')\n",
    "\n",
    "stats.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
